import threading
from queue import Queue
import setting
from utils.spider_utils import get_city, get_city_data
from spider import Spiders
from log_moudle import apispiderLog
from datasave import DataSave

logger = apispiderLog().getlogger()

# 爬虫的线程对象
class SpiderThread(threading.Thread):
    def __init__(self, threadID):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.singal = threading.Event()
        self.singal.set()

    def run(self):
        logger.debug('爬虫 {} 启动'.format(self.threadID))
        global city_queue, result_queue, result_queue_flag
        while not city_queue.empty():
            city = city_queue.get_nowait()
            try:
                result = eval(setting.SPIDERNAME)(Spiders, city=city)
            except Exception as e:
                pass
            else:
                logger.debug('SpiderThread：{} 获取 {}'.format(str(self.threadID), city))
                if result != []:
                    result_queue.put_nowait(result)
        logger.info('SpiderThread：{} 停止'.format(str(self.threadID),))
        result_queue_flag -= 1

    def pause(self):
        self.log_ctrl.AppendText("pause\n")
        self.singal.clear()

    def restart(self):
        self.log_ctrl.AppendText("continue\n")
        self.singal.set()



# 数据保存 的线程对象
class SaveThread(threading.Thread):
    def __init__(self, threadID):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.singal = threading.Event()
        self.singal.set()

    def run(self):
        logger.debug('存储 {} 启动'.format(self.threadID))
        global result_queue, result_queue_flag
        while result_queue_flag != 0 or not result_queue.empty():
            if not result_queue.empty():
                result = result_queue.get_nowait()
                eval(setting.DATASAVENAME)(DataSave,result=result)
                logger.debug('SaveThread：{} 保存： {}'.format(str(self.threadID), result))
        logger.info('SaveThread：{} 停止'.format(str(self.threadID), ))

    def pause(self):
        self.log_ctrl.AppendText("pause\n")
        self.singal.clear()

    def restart(self):
        self.log_ctrl.AppendText("continue\n")
        self.singal.set()



if __name__ == '__main__':
    num_of_threads = setting.num_of_threads
    city_queue = Queue()
    citys = get_city()
    # lt
    result_queue = Queue()  # 存储爬取结果
    result_queue_flag = setting.num_of_threads  # result_queue 标志
    # mm

    for i in citys:
        city_queue.put(i)

    threads = [SpiderThread(i+1) for i in range(num_of_threads)]
    threads.append(SaveThread(1))

    for i in range(len(threads)):
        threads[i].start()

    for i in range(len(threads)):
        threads[i].join()

